{
	"model_name_or_path": "facebook/llama-7b",
	"dataset_name_or_path": "./data",
	"output_dir": "./checkpoints/llama_lora_ckpts",
	"per_device_train_batch_size": 1,
	"gradient_accumulation_steps": 1,
	"per_device_eval_batch_size": 8,
	"eval_accumulation_steps": 16,
	"num_train_epochs": 1,
    "max_steps": 100,
	"learning_rate": 0.0003,
	"warmup_steps": 30,
	"logging_steps": 1,
	"evaluation_strategy": "no",
	"save_strategy": "steps",
	"src_length": 1024,
	"max_length": 2048,
	"bf16": true,
	"fp16_opt_level": "O2",
	"do_train": true,
	"do_eval": false,
	"disable_tqdm": true,
	"load_best_model_at_end": false,
	"eval_with_do_generation": false,
	"metric_for_best_model": "accuracy",
	"recompute": true,
	"save_total_limit": 1,
	"tensor_parallel_degree": 1,
	"pipeline_parallel_degree": 1,
	"lora": true,
	"zero_padding": false,
	"use_flash_attention": true,
	"sharding_parallel_degree": 8,
	"sharding": "stage3",
	"recompute_granularity": "full_attn",
	"autotuner_benchmark": 1,
	"benchmark": 1
}